//Includes
var fs = require('fs');
var httpAgent = require('http-agent');
var jsdom = require('jsdom');

//Database information
var TABLE_NAME = 'apkinfo';
var DB_HOST = "127.0.0.1";
var DB_PORT = 27017;

//Site specific vars
var site = "apps.opera.com";
var appPage = "en_us/free_catalog.php?soft=new";
var nextPage = "&p=";
var firstPage = 2;

var DOWNLOAD_DIR = site + "\\apks\\"; //windows NOT linux

function scrapePage(htmlbody) {
	var window = jsdom.env({
			html: htmlbody,
			done: function(errors, window) {
				var $ = require('jquery').create(window);
				var appPages = $(".title.product-details-link");
				var result = new Array();
				for(var i = 0; i < appPages.length; i++) {
					appPages[i] = $(appPages[i]).attr("href");
					result.push(appPages[i].slice(site.length + 8));
				}

				//run new agent to collect data and download .apks.
				var appAgent = httpAgent.create(site, result);

				appAgent.addListener('next', function(err, agent){
					var window = jsdom.env({
						html: appAgent.body,
						done: function(err, win) {
							var d = new Array();
							var $ = require('jquery').create(win);
							d.url = $($(".b-download")[0]).attr("href");
							var t = $($(".product-title")[0]).text();
							d.title = t.substring(0,t.lastIndexOf(" "));
							d.version = t.substring(t.lastIndexOf(" "));
							d.author = $($(".developer").children("a")).text();
							var s = $($(".stat")[0]).text();
							s = s.slice(s.indexOf("Downloads: ") + 12);
							d.downloads = s.substring(0, s.indexOf(" "));
							s = s.slice(s.indexOf("Updated: ") + 9);
							d.date = s.substring(0, s.indexOf(" "));

							//Log info and download apk
							var appdata = new appModel({
								url : d.url,
								title : d.title,
								version : d.version,
								author : d.author,
								date : d.date,
								downloads : d.downloads
							});
							appdata.save( function(error, res){
								console.log("INFO for " + d.title);
								console.log("\t URL: " + d.url);
								console.log("\t VERSION: " + d.version);
								console.log("\t AUTHOR: " + d.author);
								console.log("\t DATE: " + d.date);
								console.log("\t DOWNLOADS: " + d.downloads);
								console.log("------------------------------------");
							});
							downloadApk(d.url, d.title);

							//Next app
							appAgent.next();
						}
					});
				});

				agent.addListener('stop', function(err, agent){
					//done with this page of apps, go to next page.
					console.log("Finished this page");
					agent.next();
				});
			}
		});
}

function downloadApk(link, name){
	var request = require('request');
	request(site + link).pipe(fs.createWriteStream(DOWNLOAD_DIR + name + ".apk"));
}

//Main
var pages = new Array();
pages.push(appPage);

if(process.argv[2] > 1){
	for(var i = 0; i < process.argv[2]; i++){
		pages.push(appPage + nextPage + (firstPage + i));
	}
}

var agent = httpAgent.create(site, pages);
var mongoose = require('mongoose')
	,Schema = mongoose.Schema
	,ObjectID = Schema.ObjectID;

//Setup Mongoose
mongoose.connect(DB_HOST, TABLE_NAME, DB_PORT, {});

var appSchema = new Schema({
	url : String,
	title : String,
	version : String,
	author : String,
	date : Date,
	downloads : Number,
	lastseen : {type : Date, default : Date.now()},
	threat : {type : Number, default : 0}
});
console.log("sup");
var appModel = mongoose.model(site+'-apkdata', appSchema);

//Setup http-agent
agent.addListener('next', function(err, agent){
	
	scrapePage(agent.body);
});

agent.addListener('stop', function(err, agent){
	console.log("Finished scraping " + site);
});

//Start scraping
agent.start();
